package main.specific;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.PrintStream;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Enumeration;
import java.util.HashSet;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;

import main.Segment;
import main.SgmledSegment;
import opennlp.tools.sentdetect.SentenceDetectorME;
import opennlp.tools.sentdetect.SentenceModel;
import opennlp.tools.util.Span;

import org.mitre.jawb.io.SgmlDocument;
import org.mitre.jawb.io.SgmlElement;

import edu.stanford.nlp.ling.CoreLabel;
import edu.stanford.nlp.ling.CoreAnnotations.PartOfSpeechAnnotation;
import edu.stanford.nlp.ling.CoreAnnotations.TextAnnotation;
import edu.stanford.nlp.ling.CoreAnnotations.TokensAnnotation;
import edu.stanford.nlp.pipeline.Annotation;
import edu.stanford.nlp.pipeline.StanfordCoreNLP;
import edu.washington.cs.knowitall.extractor.mapper.ReVerbRelationDictionaryFilter;
import edu.washington.cs.knowitall.nlp.ChunkedSentence;
import edu.washington.cs.knowitall.nlp.OpenNlpChunkedSentenceParser;
import edu.washington.cs.knowitall.nlp.OpenNlpSentenceChunker;
import edu.washington.cs.knowitall.util.DefaultObjects;

public class ChunkingSpecific {

	private static SentenceDetectorME sentenceDetector;
	
	private static final String NATURAL_DISASTER = "NaturalDisaster";

	private static Hashtable<String, Integer> midAccumTextCount;

	private static ArrayList<String> midAccumTexts;

	private static Hashtable<String, Integer> midTextCount;

	private static ArrayList<String> midTexts;
	
	private static Map<String, Hashtable<String,Set<String>>> argumentsCombination = new Hashtable<String, Hashtable<String,Set<String>>>();

	private static Hashtable<String, PrintStream> printStreams = new Hashtable<String, PrintStream>();
	
	/**
	 * @param args
	 * @throws IOException 
	 */
	public static void main(String[] args) throws IOException {
		
		sentenceDetector = new SentenceDetectorME(new SentenceModel(new FileInputStream(new File("en-sent.bin"))));
		
		 // creates a StanfordCoreNLP object, with POS tagging, lemmatization, NER, parsing, and coreference resolution 
	    Properties props = new Properties();
//	    props.put("annotators", "tokenize, ssplit, pos, lemma, ner, parse, dcoref");
	    
	    props.put("annotators", "tokenize, ssplit, pos");
	    	    
	    StanfordCoreNLP pipeline = new StanfordCoreNLP(props);
		
		String fileCombinationsName = "/home/pjbarrio/Dataset/SGML-ACE/COMBINATIONS";
		
		loadFileForCombinations(new File(fileCombinationsName));
		
		String[] Datasets = main.Chunking.Datasets;
		
		for (int dataset = 0; dataset < Datasets.length; dataset++) {
			
			String[] RelationTags = getRelationTags(Datasets[dataset]);
			
			String extensionFile = getExtensionFile(Datasets[dataset]);
			
			String[] firstArgs = getFirstArgument(Datasets[dataset]);
			
			for (int first = 0; first < firstArgs.length; first++) {
				
				String[] secondArgs = getSecondArgument(Datasets[dataset],firstArgs[first]);
				
				for (int second = 0; second < secondArgs.length; second++) {
					
					int[] splits = getSplits(Datasets[dataset]);
										
					for (int spl = 0; spl < splits.length; spl++) {
							
						String[] tags = new String[2];
						tags[0] = firstArgs[first];
						tags[1] = secondArgs[second];
						
						String typeOfExperiment = "test";
						
						String foldersFile = "taggedtestingDirectory";

						String frequentFile = "old_freq_rel.txt.gz";
						
						boolean separateClasses = true;
						
						
						String newFrequentFile = getFrequentFile(Datasets[dataset],splits[spl],tags[0],tags[1],typeOfExperiment,"freq_rel");
						
						String newFrequentFileAccumulated = getFrequentFile(Datasets[dataset],splits[spl],tags[0],tags[1],typeOfExperiment,"acc_freq_rel");
												
//						String outputFile = rootDirectory + spl+"/" + Datasets[dataset] + "_" + tags[0] + "_" + tags[1] + "_" + splits[spl] + ".reverb." + typeOfExperiment;
//			
//						String newFrequentFile = rootDirectory+spl+"/"+Datasets[dataset]+ "_" + tags[0] + "_" + tags[1] + "_" + splits[spl] + "_acc_freq_rel." + typeOfExperiment + ".txt.gz";
//				
//						String newFrequentFileAccumulated = rootDirectory+spl+"/"+Datasets[dataset] + "_" + tags[0] + "_" + tags[1] + "_" + splits[spl] + "_freq_rel." + typeOfExperiment + ".txt.gz";
						
//						System.setOut(new PrintStream(new File(outputFile)));
						
//						System.setErr(new PrintStream(new File(outputFile + ".error")));
						
						File directory = new File(getFilesFolder(Datasets[dataset],splits[spl],foldersFile));
						
						File[] files = directory.listFiles();
				
				//		Arrays.sort(files);
				
						midAccumTextCount = new Hashtable<String, Integer>();
						
						midAccumTexts = new ArrayList<String>();
								
						midTextCount = new Hashtable<String, Integer>();
						
						midTexts = new ArrayList<String>();
						
						loadFrequentRelations(frequentFile);
						
						for (File file : files) {
							
							if (!file.getName().endsWith(extensionFile))
								continue;
				
							System.err.println(file.getName());
							
							SgmlDocument sgmlDocument = new SgmlDocument(new FileReader(file));
							
							Hashtable<String, List<SgmlElement>> sgmlElements = getSgmlElements(sgmlDocument,tags);
							
							Hashtable<String, List<SgmlElement>> sgmlRelationElements = getSgmlElements(sgmlDocument, RelationTags);
							
							List<Segment> sentences = getRelevantSentences(sgmlDocument,firstArgs[first],secondArgs[second],sgmlElements,tags); 
				
							for (Segment segment : sentences) {
								
								List<SgmledSegment> combinations = generateCombinations(segment,firstArgs[first],secondArgs[second],sgmlElements,tags);
								
								Hashtable<String, List<SgmlElement>> filteredRelations = filter(sgmlRelationElements, segment, RelationTags);
								
								for (SgmledSegment sgmledSegment : combinations) {
									
									// create an empty Annotation just with the given text
								    Annotation sentence = new Annotation(sgmledSegment.getText());
								    
								    // run all Annotators on this text
								    pipeline.annotate(sentence);
									
									SgmlElement first_element = sgmledSegment.getRelevantSgmlElement(1);
									
									SgmlElement second_element = sgmledSegment.getRelevantSgmlElement(2);
									
									if (!rightOrder(first_element,second_element)){
										
										SgmlElement auxiliarElement = first_element;
										first_element = second_element;
										second_element = auxiliarElement;
										
									}
									
									if (!acceptable(first_element,second_element))
										continue;
									
									List<CoreLabel> tokens = sentence.get(TokensAnnotation.class);
																
									Integer[] firstText = generateArray(first_element,tokens, sgmledSegment.getOffset());
									
									Integer[] secondText = generateArray(second_element,tokens, sgmledSegment.getOffset());
									
									if (firstText[firstText.length-1] >= (secondText[0] - 1)) //no words in the middle.
										continue;
									
									String[] words = new String[tokens.size()];
									
									String[] pos = new String[tokens.size()];
									
									int cl = getClass(sgmledSegment, filteredRelations, RelationTags[0],RelationTags[1]);
									
									PrintStream ps = getPrintStream(tags[0],tags[1],Datasets[dataset],splits[spl],cl>0,typeOfExperiment,separateClasses);
									
									int index = 0;
									
							    	for (CoreLabel token: tokens){
							    		
							    		String word = token.get(TextAnnotation.class);
							    		
							    		words[index] = word;
							    		
							    		index++;
							    		
							    		ps.print(word + " ");
							    		
							    	}
									
							    	ps.println();
									
							    	index = 0;
							    	
									for (CoreLabel token: tokens){
										
										String post = token.get(PartOfSpeechAnnotation.class);
										
										pos[index] = post;
										
										index++;
										
										ps.print(post + " ");
										
									}
									
									String[] chunks = DefaultObjects.getDefaultChunker().chunk(words, pos);
									
									ChunkedSentence chunkedSentence = new ChunkedSentence(words, pos, chunks);
									
									ps.println();

									ps.println(chunkedSentence.getNpChunkTagsAsString());
									
									ps.println(getText(firstText,tokens));
									
									ps.println(firstText[0] + " " + firstText.length);
									
									String midText = getMiddleText(firstText,secondText,tokens);
									
									ps.println(midText);
									
									updateMidTextCount(midText);
									
									int mid = secondText[0]-(firstText[firstText.length-1] + 1);
									
									int start = firstText[firstText.length-1] + 1;
									
									ps.println(start + " " + mid);
									
									ps.println(getText(secondText,tokens));
									
									ps.println(secondText[0] + " " + secondText.length);
								
									ps.println(cl);
								}
								
								
							}
									
						}
						
						Collections.sort(midAccumTexts, new Comparator<String>(){
				
							@Override
							public int compare(String mT1, String mT2) {
								
								return midAccumTextCount.get(mT2).compareTo(midAccumTextCount.get(mT1));
								
							}
							
						});
						
						BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new GZIPOutputStream(new FileOutputStream(new File(newFrequentFileAccumulated)))));
						
						for (String midText : midAccumTexts) {
							
							bw.write(midAccumTextCount.get(midText) + "\t" + midText + "\n");
							
						}
						
						bw.close();
						
						Collections.sort(midTexts, new Comparator<String>(){
				
							@Override
							public int compare(String mT1, String mT2) {
								
								return midTextCount.get(mT2).compareTo(midTextCount.get(mT1));
								
							}
							
						});
						
						bw = new BufferedWriter(new OutputStreamWriter(new GZIPOutputStream(new FileOutputStream(new File(newFrequentFile)))));
						
						for (String midText : midTexts) {
							
							bw.write(midTextCount.get(midText) + "\t" + midText + "\n");
							
						}
						
						bw.close();
					}	

					
				}

			}
			
		}

		closeAllPrintStreams();
		
	}

	public static String getFrequentFile(String dataset, int split, String tag1,
			String tag2, String typeOfExperiment, String freq_file) {
		
		return getDirectory(dataset)+split+"/"+dataset+ "_" + tag1 + "_" + tag2 + "_" + split + "_" + freq_file + "." + typeOfExperiment + ".txt.gz";
		
	}

//	public static String getPrefixFile(String dataset, int split, String tag0,
//			String tag1, String typeOfExperiment) {
//		
//		return getDirectory(dataset) + split+"/" + dataset+ "_" + tag0 + "_" + tag1 + "_" + split + ".reverb." + typeOfExperiment;
//		
//	}

	private static String getFilesFolder(String dataset, int split,
			String foldersFile) {
		
		return getDirectory(dataset) + split +"/"+ foldersFile +"/";
		
	}

	private static void closeAllPrintStreams() {
		
		for(Enumeration<PrintStream> e = printStreams.elements(); e.hasMoreElements();){
			e.nextElement().close();
		}
		
	}

	private static PrintStream getPrintStream(String gid1, String gid2, String dataset, int split, boolean classification, String fileFor, boolean separatedClass) throws FileNotFoundException{
		
		String file = getFormattedFile(gid1,gid2,dataset,split,classification,fileFor,separatedClass);
		
		PrintStream ret = printStreams.get(file);
		
		if (ret == null){
			ret = new PrintStream(new File(file));
			printStreams.put(file,ret);
		}
		
		return ret;
	}

	public static String[] getFirstArgument(String dataset) {
		return argumentsCombination.get(dataset).keySet().toArray(new String[0]);
	}

	public static String[] getSecondArgument(String dataset, String first) {
		
		return argumentsCombination.get(dataset).get(first).toArray(new String[0]);
		
	}
	
		
	private static String getMiddleText(Integer[] firstText,
			Integer[] secondText, List<CoreLabel> tokens) {
		
		
		String ret = "";
		
		int begin = firstText[firstText.length-1] + 1;
		
		int end = secondText[0];
		
		for (int i = begin; i < end; i++) {
		
			ret = ret + " " + tokens.get(i).get(TextAnnotation.class);
			
		}
		
		return ret.substring(1);
		
	}

	public static String getSplittedFormatedFile(String generatedPrefix,
			int split) {
	
		return generatedPrefix + "." + split;
	
	}
	
	private static Integer[] generateArray(SgmlElement element,
			List<CoreLabel> tokens, int offset) {
		
		ArrayList<Integer> ret = new ArrayList<Integer>();
		
		int elementBegin = element.getStart() - offset;
		
		int elementEnd = element.getEnd() - offset;
		
		int index = 0;
		
		for (CoreLabel token : tokens) {

			int begin = token.beginPosition();
			
			int end = token.endPosition();

//			System.err.println(elementBegin + " - " + elementEnd + " - " + begin + " - " + end + " - " + token.get(TextAnnotation.class));
			
			if (end <= elementBegin || begin >= elementEnd){
				
				if (begin >= elementEnd){
					break;
				}
				
				index++;
				continue;
				
			}
			
	        ret.add(new Integer(index));        
			
			index++;
		}

		
		
		return ret.toArray(new Integer[0]);
		
	}

	private static boolean rightOrder(SgmlElement first_element,
			SgmlElement second_element) {
		
		return first_element.getStart() < second_element.getStart() && first_element.getEnd() < second_element.getEnd();
		
	}

	private static boolean acceptable(SgmlElement first_element,
			SgmlElement second_element) {
		
		return first_element.getEnd() <= second_element.getStart();
		
	}

	private static String getExtensionFile(String relation) {
		
		if (relation.equals(NATURAL_DISASTER)){
			return ".txt";
		}
	
		return ".sgm";
		
	}

	public static String getDirectory(String relation) {
		
		if (relation.equals(NATURAL_DISASTER)){
			return "/home/pjbarrio/Dataset/NaturalDisaster/RelatedData/reverbData/";
		}
	
		return "/home/pjbarrio/Dataset/SGML-ACE-COREF/" + relation + "/";
		
		
	}

	public static void loadFileForCombinations(File file) throws IOException {
		
		BufferedReader br = new BufferedReader(new FileReader(file));
		
		String line = br.readLine();
		
		while (line!=null){
			
			System.out.println(line);
			
			String[] split = line.split(" - ");
			
			String relation = split[0];
			
			String first = split[1];
			
			String second = split[2];
			
			Hashtable<String, Set<String>> args = argumentsCombination.get(relation);
			
			if (args == null){
				
				args = new Hashtable<String, Set<String>>();
				
			}
			
			Set<String> firstArg = args.get(first);
			
			if (firstArg == null){
				
				firstArg = new HashSet<String>();
				
			}
			
			firstArg.add(second);			
			
			args.put(first,firstArg);
			
			argumentsCombination.put(relation, args);
			
			line = br.readLine();
		
		}
		
		br.close();
		
	}

	public static int[] getSplits(String relation) {
		
		if (relation.equals(NATURAL_DISASTER)){
			int[] ret = {0,1,2,3,4,5,6,7,8,9};
			return ret;
		}
	
		int[] ret = {0};
		
		return ret;
		
	}

	private static String[] getRelationTags(String relation) {
		
		if (relation.equals(NATURAL_DISASTER)){
			String[] ret = {"atLocation","locationOf"};
			return ret;
		}
	
		String[] ret = {relation + "First",relation + "Second"};
		
		return ret;
	}

	private static void loadFrequentRelations(String file) throws IOException {
		
		BufferedReader br = new BufferedReader(new InputStreamReader(
				new GZIPInputStream(ReVerbRelationDictionaryFilter.class.getClassLoader().getResourceAsStream(file))));
		
		String line = br.readLine();
		
		while (line != null){
			
			String[] split = line.split("\t");
			
			midAccumTexts.add(split[1]);
			
			midAccumTextCount.put(split[1], Integer.valueOf(split[0]));
		
			line = br.readLine();
			
		}
		
		br.close();
		
	}

	private static void updateMidTextCount(String midText) {
		
		Integer freqAccum = midAccumTextCount.get(midText);
		
		if (freqAccum == null){
			midAccumTextCount.put(midText, 1);
			midAccumTexts.add(midText);
						
		}else{
			midAccumTextCount.put(midText, freqAccum+1);
			
		}

		Integer freq = midTextCount.get(midText);
		
		if (freq == null){
			midTextCount.put(midText, 1);
			midTexts.add(midText);
			
		}else{
			midTextCount.put(midText, freq+1);
		}

		
	}


	private static String getText(Integer[] textIndex, List<CoreLabel> tokens) {
		
		String ret = "";
		
		for (Integer index : textIndex) {
			
			ret = ret + " " + tokens.get(index).get(TextAnnotation.class);
			
		}
		
		return ret.substring(1);
		
	}

	private static int getClass(SgmledSegment sgmledSegment,
			Hashtable<String, List<SgmlElement>> filteredRelations, String firstTag, String secondTag) {
		// returns 1 if the entities are related. 0 otherwise.
		
		SgmlElement relevantElement1 = sgmledSegment.getRelevantSgmlElement(1);
		
		SgmlElement relevantElement2 = sgmledSegment.getRelevantSgmlElement(2);
				
		List<SgmlElement> relevantRelationFirst = new ArrayList<SgmlElement>();
		
		List<SgmlElement> relevantRelationSecond = new ArrayList<SgmlElement>();
		
		List<SgmlElement> relations = filteredRelations.get(firstTag);
		
		for (SgmlElement relationSgmlElement : relations) {
			
			if (matches(relationSgmlElement,relevantElement1)){
				relevantRelationFirst.add(relationSgmlElement);
			}
			
		}

		
		relations = filteredRelations.get(secondTag);
		
		for (SgmlElement relationSgmlElement : relations) {
			
			if (matches(relationSgmlElement,relevantElement2)){
				relevantRelationSecond.add(relationSgmlElement);
			}
			
		}
		
		
		if (relevantRelationFirst.size() > 0 && relevantRelationSecond.size()>0){
			
			if (sameRelation(relevantRelationFirst,relevantRelationSecond)){
				return 1;
			}
			
		}
		
		return 0;
	}

	private static boolean sameRelation(List<SgmlElement> relevantRelationFirst, List<SgmlElement> relevantRelationSecond) {
		
		for (SgmlElement sgmlElement : relevantRelationFirst) {
			
			String attribute = sgmlElement.getAttribute("id");
			
			for (SgmlElement sgmlElement2 : relevantRelationSecond) {
				
				String attribute2 = sgmlElement.getAttribute("id");
				
				if (attribute.equals(attribute2)){
					return true;
				}
				
			}
			
		}
		
		return false;
		
	}

	private static boolean matches(SgmlElement relationSgmlElement,
			SgmlElement sgmlElement) {
		
		return (relationSgmlElement.getStart() >= sgmlElement.getStart() && relationSgmlElement.getEnd() <= sgmlElement.getEnd()) 
				|| (sgmlElement.getStart()>=relationSgmlElement.getStart() && sgmlElement.getEnd() <= relationSgmlElement.getEnd());
		
	}

	private static List<SgmledSegment> generateCombinations(Segment segment, String tag1, String tag2, Hashtable<String, List<SgmlElement>> sgmlElements, String[] tags) {
		
		List<SgmledSegment> ret = new ArrayList<SgmledSegment>();
		
		Hashtable<String, List<SgmlElement>> containedSgmlElements = filter(sgmlElements,segment,tags);
	
		List<SgmlElement> sorted = sort(containedSgmlElements);

		List<SgmlElement> tag_0 = containedSgmlElements.get(tag1);

		List<SgmlElement> tag_1 = containedSgmlElements.get(tag2);
				
		for (SgmlElement sgmlElement_0 : tag_0) {
			
			for (SgmlElement sgmlElement_1 : tag_1) {
				
				SgmledSegment aux = new SgmledSegment(segment.getText(), segment.getOffset(), sorted);
				
				aux.addSgmlElement(sgmlElement_0,1);
				
				aux.addSgmlElement(sgmlElement_1,2);
				
				ret.add(aux);
				
			}
			
		}
				
				
		
		return ret;
		
	}

	private static List<SgmlElement> sort(
			Hashtable<String, List<SgmlElement>> containedSgmlElements) {
		
		List<SgmlElement> totalSgmlElements = new ArrayList<SgmlElement>();
		
		for(Enumeration<String> e = containedSgmlElements.keys();e.hasMoreElements();){
			totalSgmlElements.addAll(containedSgmlElements.get(e.nextElement()));
		}

		Collections.sort(totalSgmlElements, new Comparator<SgmlElement>() {

			@Override
			public int compare(SgmlElement arg0, SgmlElement arg1) {
				
				return Integer.valueOf(arg0.getStart()).compareTo(arg1.getStart());
				
			}
		});
		
		return totalSgmlElements;
	}

	private static Hashtable<String, List<SgmlElement>> filter(
			Hashtable<String, List<SgmlElement>> sgmlElements, Segment segment,
			String[] tags) {
		
		Hashtable<String, List<SgmlElement>> ret = new Hashtable<String, List<SgmlElement>>();
		
		for (int i = 0; i < tags.length; i++) {
			
			List<SgmlElement> elements = sgmlElements.get(tags[i]);
			
			ArrayList<SgmlElement> aux = new ArrayList<SgmlElement>();
			
			for (SgmlElement sgmlElement : elements) {
				
				if (contains(segment, sgmlElement)){
					aux.add(sgmlElement);
				}
				
			}
			
			ret.put(tags[i], aux);
			
		}
		
		return ret;
	}

	private static List<Segment> getRelevantSentences(SgmlDocument sgmlDocument, String tag1, String tag2, Hashtable<String, List<SgmlElement>> sgmlElements, String[] Tags) throws FileNotFoundException, IOException {
		
		String plainText = sgmlDocument.getSignalText();
		
		List<Segment> segments = getSegments(plainText);
		
		List<Segment> sentences = generateSentences(segments);
		
		List<Segment> fixedSentences = fixSentences(sentences,sgmlElements,Tags);
		
		List<Segment> filteredSenteces = filterSentences(sgmlElements,fixedSentences,tag1,tag2);
		
		return filteredSenteces;
	}

	private static List<Segment> fixSentences(List<Segment> segments,
			Hashtable<String, List<SgmlElement>> sgmlElements, String[] tags) {

		List<Segment> ret = new ArrayList<Segment>();

		for (int j = 0; j < segments.size(); j++) {
			
			Segment seg = segments.get(j);
			
			boolean changed = false;
			
			for (int i = 0; i < tags.length && !changed; i++) {
				
				List<SgmlElement> se = sgmlElements.get(tags[i]);
				
				for (SgmlElement sgmlElement : se) {
					
					if (contains(seg, sgmlElement) && !containsItAll(seg, sgmlElement)){
						
						String link = "";
						
						int initSec = segments.get(j+1).getOffset();
						int endPrev = seg.getOffset() + seg.getText().length();
						
						int add = initSec - endPrev;
						
						for (int k = 0; k < add; k++) {
						
							link = link + " ";
						
						}
						
						segments.set(j+1, new Segment(seg.getText() + link + segments.get(j+1).getText(), seg.getOffset()));
						
						changed = true;
						
						break;
						
					}
					
				}
				
			}
			
			if (!changed)
				ret.add(seg);
			
		}
		
		return ret;
		
	}

	private static List<Segment> filterSentences(
			Hashtable<String, List<SgmlElement>> sgmlElements,
			List<Segment> sentences, String tag1, String tag2) {
		// return sentences that have both tags.
		
		ArrayList<Segment> ret = new ArrayList<Segment>();
		
		for (Segment sentence : sentences) {
					
			if (contains(sentence,sgmlElements.get(tag1)) && contains(sentence,sgmlElements.get(tag2))){
				ret.add(sentence);
			}
			
		}
	
		return ret;
	}

	private static boolean contains(Segment sentence, List<SgmlElement> list) {
		
		for (SgmlElement sgmlElement : list) {
			
			if (contains(sentence,sgmlElement)){
				return true;
			}
						
		}
		
		return false;
	}

	private static boolean containsItAll(Segment sentence, SgmlElement sgmlElement) {

		int start = sentence.getOffset();
		int end = start + sentence.getText().length();

		if (sgmlElement.getStart() >= start && sgmlElement.getEnd() <= end){
			
			return true;
			
		}
		
		return false;
		
	}
	
	private static boolean contains(Segment sentence, SgmlElement sgmlElement) {
		int start = sentence.getOffset();
		int end = start + sentence.getText().length();

		if (sgmlElement.getStart() >= start && sgmlElement.getStart() <= end){
			
			return true;
			
		}
		
		return false;
		
	}

	private static Hashtable<String, List<SgmlElement>> getSgmlElements(
			SgmlDocument sgmlDocument, String[] tags) {
		
		Hashtable<String, List<SgmlElement>> ret = new Hashtable<String, List<SgmlElement>>();
		
		for (int i = 0; i < tags.length; i++) {
			
			ArrayList<SgmlElement> aux = new ArrayList<SgmlElement>();
			
			for (Iterator iterator = sgmlDocument.iterator(); iterator.hasNext();) {
				
				SgmlElement sgmlElement = (SgmlElement) iterator.next();
				
				if (sgmlElement.getGid().equals(tags[i])){
					
					aux.add(sgmlElement);
					
				}
				
			}
			
			ret.put(tags[i],aux);
			
		}
		
		return ret;
		
	}

	private static List<Segment> generateSentences(List<Segment> contentParagraph) {
		
		ArrayList<Segment> ret = new ArrayList<Segment>();
		
		for (Segment segment : contentParagraph){
			
			ret.addAll(getSentences(segment));
			
		}

		return ret;
		
	}

	private static Collection<? extends Segment> getSentences(Segment segment) {
		
		ArrayList<Segment> ret = new ArrayList<Segment>();
		
		String sentences[] = sentenceDetector.sentDetect(segment.getText());
			
		Span[] spans = sentenceDetector.sentPosDetect(segment.getText());
		
		for (int i = 0; i < sentences.length; i++) {
			
			ret.add(new Segment(sentences[i],segment.getOffset() + spans[i].getStart()));
			
		}

		return ret;
		
	}

	
	private static List<Segment> getSegments(String text) throws IOException {
		
		BufferedReader br = new BufferedReader(new StringReader(text));
		
		ArrayList<Segment> ret = new ArrayList<Segment>();
		
	    String acum = "";
	    
	    String line = br.readLine();
	     
	    int offset = 0;
	    
	    int newOffset = 0;
	    
	    while (line != null){
	    	
	    	if (line.trim().equals("")){
	    		
	    		if (!acum.trim().equals("")){
	    				    			
	    			Segment s = new Segment(acum,offset);
	    			
	    			ret.add(s);
	    		
	    		}
	    		
	    		acum = "";
	    	
	    	} else {
	    		
	    		if (acum.trim().equals("")){  //First
	    			
	    			acum = line;
	    			
	    			offset = newOffset;
	    			
	    		} else {
		    		acum = acum + " " + line;
	    		}
	    		
	    	}
	    	
	    	newOffset += line.length() + 1;
	    	
	    	line = br.readLine();
	    
	    }
	    
	    if (!acum.trim().equals("")){
	    	
	    	Segment s = new Segment(acum,offset);
	    	
	    	ret.add(s);
	    	
	    }
	    
	    br.close();
	    
	    return ret;
		
	}

	public static String getFormattedFile(String gid, String gid2,
			String dataset, int split, boolean classification, String fileFor, boolean separateClasses) {
		
		String outputFilePrefix = getDirectory(dataset) + split+ "/" + dataset;
				
		String id = getSuffix(gid,gid2,split,classification,separateClasses);

		return outputFilePrefix + "_" + id + ".reverb." + fileFor;
		
	}

	private static String getSuffix(String gid, String gid2, int split, boolean classification, boolean separateClasses) {
		
		if (separateClasses)
			return gid + "_" + gid2 + "_" + split + "_" + classification;
		
		return gid + "_" + gid2 + "_" + split;
	
	}
	
	
}
